import time
from setting import *
# from tools import *
from lxml import html
from util.data_utils import *
import time
import random

def get_url_data(key_url, group_name):
    print(getCurTime(), f"【{group_name} 开始】")

    response = ownRequest(key_url)
    tree = html.fromstring(response.text)
    # xpath获取所有title标签
    urls = tree.xpath("//*[@id='group-topics']/div[2]/table/tr[position()>1]/td[1]/a/@href")
    
    for url in urls:
        # unique_id = urlparse(url).path
        unique_id = url.split("/")[-2]
        
        # https://www.douban.com/group/topic/327504110/?_spm_id=MTQ3NDM5ODc3&_i=9523837XjFQJ4B,9524813XjFQJ4B
        if R.sismember("unique_set_douban", unique_id):
            print(f"已经提取过该文章：{unique_id}")
            continue
        
        data = fetch_data(group_name, unique_id,url)
        # 主动推过去的文章 为0
        data["push_task_id"] = 0
        
        # 保存unique_id
        if toDatabase(data): 
            R.sadd("unique_set_douban", unique_id)

    print(getCurTime(), f"【{group_name} 结束】")
    #logger.info(f"【{group_name} 结束】")

    time.sleep(20)

def main():
    # 死循环
    while True:
        try:
            print(f"\n{getCurTime()} === 开始新一轮爬取 ===")
            
            # 创建并启动多个线程
            for items in key_url_lis:
                url = items[0]
                group_name = items[1]
                get_url_data(url, group_name)
                sleep_time = random.uniform(1, 5)
                print(f"等待 {sleep_time:.2f} 秒...")
                time.sleep(sleep_time)
            
            # 每轮结束后等待较长时间
            round_sleep = random.uniform(30, 60)  # 30-60秒
            print(f"{getCurTime()} 本轮结束，等待 {round_sleep:.2f} 秒后开始下一轮...")
            time.sleep(round_sleep)
            
        except KeyboardInterrupt:
            print(f"{getCurTime()} 用户中断程序")
            break
        except Exception as e:
            print(f"{getCurTime()} 发生错误: {e}, 10秒后重试...")
            time.sleep(10)

if __name__ == "__main__":
    # 买组  拼组
    key_url_lis = [
        ["https://www.douban.com/group/698716/?tab=51757", "买组"],
        ["https://www.douban.com/group/536786/?tab=47726", "拼组"]
    ]
    
    main()
